{
 "cells": [
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-17T08:11:15.148252Z",
     "start_time": "2025-09-17T08:11:15.106250Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import nest_asyncio\n",
    "\n",
    "nest_asyncio.apply()"
   ],
   "id": "7eec933bbadb28e2",
   "outputs": [],
   "execution_count": 1
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-17T08:17:49.489199Z",
     "start_time": "2025-09-17T08:16:57.789207Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import os\n",
    "from llama_index.core import Settings\n",
    "os.environ['DEEPSEEK_API_KEY']='sk-f5f3487896554276aaf657fe9b9661a7'\n",
    "from llama_index.llms.deepseek import DeepSeek\n",
    "from llama_index.core.embeddings import resolve_embed_model\n",
    "\n",
    "base_embed_model = resolve_embed_model(\"local:D:/pythonProject17/transformers/model_em/BAAI/bge-small-en-v1.5\")\n",
    "llm=DeepSeek(model='deepseek-chat')\n",
    "\n",
    "Settings.llm=llm\n",
    "Settings.embed_model=base_embed_model"
   ],
   "id": "4bd30fccd24ef8d8",
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 16:17:33,515 - INFO - PyTorch version 2.5.1+cu121 available.\n",
      "2025-09-17 16:17:36,382 - INFO - Load pretrained SentenceTransformer: D:/pythonProject17/transformers/model_em/BAAI/bge-small-en-v1.5\n"
     ]
    }
   ],
   "execution_count": 8
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-17T08:11:34.746628Z",
     "start_time": "2025-09-17T08:11:34.734631Z"
    }
   },
   "cell_type": "code",
   "source": [
    "wiki_titles = [\n",
    "    \"Toronto\",\n",
    "    \"Seattle\",\n",
    "    \"Chicago\",\n",
    "    \"Boston\",\n",
    "    \"Houston\",\n",
    "    \"Tokyo\",\n",
    "    \"Berlin\",\n",
    "    \"Lisbon\",\n",
    "]"
   ],
   "id": "51ed1a15aaf28b71",
   "outputs": [],
   "execution_count": 2
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-17T08:16:17.187197Z",
     "start_time": "2025-09-17T08:16:07.249923Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from pathlib import Path\n",
    "\n",
    "import requests\n",
    "\n",
    "for title in wiki_titles:\n",
    "    response = requests.get(\n",
    "        \"https://en.wikipedia.org/w/api.php\",\n",
    "        params={\n",
    "            \"action\": \"query\",\n",
    "            \"format\": \"json\",\n",
    "            \"titles\": title,\n",
    "            \"prop\": \"extracts\",\n",
    "            # 'exintro': True,\n",
    "            \"explaintext\": True,\n",
    "        },\n",
    "       headers = {\n",
    "    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36'\n",
    "}\n",
    "    ).json()\n",
    "    page = next(iter(response[\"query\"][\"pages\"].values()))\n",
    "    wiki_text = page[\"extract\"]\n",
    "\n",
    "    data_path = Path(\"data\")\n",
    "    if not data_path.exists():\n",
    "        Path.mkdir(data_path)\n",
    "\n",
    "    with open(data_path / f\"{title}.txt\", \"w\") as fp:\n",
    "        fp.write(wiki_text)"
   ],
   "id": "9fb61dc34e6c5a02",
   "outputs": [],
   "execution_count": 5
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-17T08:16:35.207505Z",
     "start_time": "2025-09-17T08:16:25.584379Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from llama_index.core import SimpleDirectoryReader\n",
    "\n",
    "# Load all wiki documents\n",
    "city_docs = {}\n",
    "for wiki_title in wiki_titles:\n",
    "    city_docs[wiki_title] = SimpleDirectoryReader(\n",
    "        input_files=[f\"data/{wiki_title}.txt\"]\n",
    "    ).load_data()"
   ],
   "id": "79370486bf637d77",
   "outputs": [],
   "execution_count": 6
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-17T08:18:02.130409Z",
     "start_time": "2025-09-17T08:18:02.095412Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# define descriptions/choices for tools\n",
    "city_descs_dict = {}\n",
    "# these choices will be passed to the router selector\n",
    "choices = []\n",
    "choice_to_id_dict = {}\n",
    "\n",
    "for idx, wiki_title in enumerate(wiki_titles):\n",
    "    vector_desc = (\n",
    "        \"Useful for questions related to specific aspects of\"\n",
    "        f\" {wiki_title} (e.g. the history, arts and culture,\"\n",
    "        \" sports, demographics, or more).\"\n",
    "    )\n",
    "    summary_desc = (\n",
    "        \"Useful for any requests that require a holistic summary\"\n",
    "        f\" of EVERYTHING about {wiki_title}. For questions about\"\n",
    "        \" more specific sections, please use the vector_tool.\"\n",
    "    )\n",
    "    doc_id_vector = f\"{wiki_title}_vector\"\n",
    "    doc_id_summary = f\"{wiki_title}_summary\"\n",
    "    city_descs_dict[doc_id_vector] = vector_desc\n",
    "    city_descs_dict[doc_id_summary] = summary_desc\n",
    "\n",
    "    choices.extend([vector_desc, summary_desc])\n",
    "    choice_to_id_dict[idx * 2] = f\"{wiki_title}_vector\"\n",
    "    choice_to_id_dict[idx * 2 + 1] = f\"{wiki_title}_summary\""
   ],
   "id": "cec28af5e3876c1a",
   "outputs": [],
   "execution_count": 9
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-17T08:18:17.421048Z",
     "start_time": "2025-09-17T08:18:17.395047Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from llama_index.llms.openai import OpenAI\n",
    "from llama_index.core import PromptTemplate\n",
    "\n",
    "# llm = OpenAI(model_name=\"gpt-3.5-turbo\")\n",
    "\n",
    "summary_q_tmpl = \"\"\"\\\n",
    "You are a summary question generator. Given an existing question which asks for a summary of a given topic, \\\n",
    "generate {num_vary} related queries that also ask for a summary of the topic.\n",
    "\n",
    "For example, assuming we're generating 3 related questions:\n",
    "Base Question: Can you tell me more about Boston?\n",
    "Question Variations:\n",
    "Give me an overview of Boston as a city.\n",
    "Can you describe different aspects of Boston, from the history to the sports scene to the food?\n",
    "Write a concise summary of Boston; I've never been.\n",
    "\n",
    "Now let's give it a shot!\n",
    "\n",
    "Base Question: {base_question}\n",
    "Question Variations:\n",
    "\"\"\"\n",
    "summary_q_prompt = PromptTemplate(summary_q_tmpl)"
   ],
   "id": "cd5c24399868343a",
   "outputs": [],
   "execution_count": 10
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-17T08:18:36.649613Z",
     "start_time": "2025-09-17T08:18:34.976142Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from collections import defaultdict\n",
    "from llama_index.core.evaluation import DatasetGenerator\n",
    "from llama_index.core.evaluation import EmbeddingQAFinetuneDataset\n",
    "from llama_index.core.node_parser import SimpleNodeParser\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "\n",
    "def generate_dataset(\n",
    "    wiki_titles,\n",
    "    city_descs_dict,\n",
    "    llm,\n",
    "    summary_q_prompt,\n",
    "    num_vector_qs_per_node=2,\n",
    "    num_summary_qs=4,\n",
    "):\n",
    "    # generate dataset from each wikipedia page\n",
    "    queries = {}\n",
    "    corpus = {}\n",
    "    relevant_docs = defaultdict(list)\n",
    "    for idx, wiki_title in enumerate(tqdm(wiki_titles)):\n",
    "        doc_id_vector = f\"{wiki_title}_vector\"\n",
    "        doc_id_summary = f\"{wiki_title}_summary\"\n",
    "        corpus[doc_id_vector] = city_descs_dict[doc_id_vector]\n",
    "        corpus[doc_id_summary] = city_descs_dict[doc_id_summary]\n",
    "\n",
    "        # generate questions for semantic search\n",
    "        node_parser = SimpleNodeParser.from_defaults()\n",
    "        nodes = node_parser.get_nodes_from_documents(city_docs[wiki_title])\n",
    "\n",
    "        dataset_generator = DatasetGenerator(\n",
    "            nodes,\n",
    "            llm=llm,\n",
    "            num_questions_per_chunk=num_vector_qs_per_node,\n",
    "        )\n",
    "        doc_questions = dataset_generator.generate_questions_from_nodes(\n",
    "            num=len(nodes) * num_vector_qs_per_node\n",
    "        )\n",
    "        for query_idx, doc_question in enumerate(doc_questions):\n",
    "            query_id = f\"{wiki_title}_{query_idx}\"\n",
    "            relevant_docs[query_id] = [doc_id_vector]\n",
    "            queries[query_id] = doc_question\n",
    "\n",
    "        # generate questions for summarization\n",
    "        base_q = f\"Give me a summary of {wiki_title}\"\n",
    "        fmt_prompt = summary_q_prompt.format(\n",
    "            num_vary=num_summary_qs,\n",
    "            base_question=base_q,\n",
    "        )\n",
    "        raw_response = llm.complete(fmt_prompt)\n",
    "        raw_lines = str(raw_response).split(\"\\n\")\n",
    "        doc_summary_questions = [l for l in raw_lines if l != \"\"]\n",
    "        print(f\"[{idx}] Original Question: {base_q}\")\n",
    "        print(\n",
    "            f\"[{idx}] Generated Question Variations: {doc_summary_questions}\"\n",
    "        )\n",
    "        for query_idx, doc_summary_question in enumerate(\n",
    "            doc_summary_questions\n",
    "        ):\n",
    "            query_id = f\"{wiki_title}_{query_idx}\"\n",
    "            relevant_docs[query_id] = [doc_id_summary]\n",
    "            queries[query_id] = doc_summary_question\n",
    "\n",
    "    return EmbeddingQAFinetuneDataset(\n",
    "        queries=queries, corpus=corpus, relevant_docs=relevant_docs\n",
    "    )"
   ],
   "id": "bb13128215f2ba34",
   "outputs": [],
   "execution_count": 11
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-17T08:21:37.888401Z",
     "start_time": "2025-09-17T08:18:53.325190Z"
    }
   },
   "cell_type": "code",
   "source": [
    "dataset = generate_dataset(\n",
    "    wiki_titles,\n",
    "    city_descs_dict,\n",
    "    llm,\n",
    "    summary_q_prompt,\n",
    "    num_vector_qs_per_node=4,\n",
    "    num_summary_qs=5,\n",
    ")"
   ],
   "id": "4eb02e8215a93f95",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "  0%|          | 0/8 [00:00<?, ?it/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "ddfb698e00e14b4a8b1be71df59eea73"
      }
     },
     "metadata": {},
     "output_type": "display_data",
     "jetTransient": {
      "display_id": null
     }
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\HASEE\\AppData\\Local\\Temp\\ipykernel_4504\\719365437.py:30: DeprecationWarning: Call to deprecated class DatasetGenerator. (Deprecated in favor of `RagDatasetGenerator` which should be used instead.)\n",
      "  dataset_generator = DatasetGenerator(\n",
      "2025-09-17 16:18:59,191 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:18:59,205 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:18:59,218 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:18:59,224 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:18:59,228 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:18:59,287 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:18:59,302 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:18:59,336 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:18:59,341 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:18:59,345 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:18:59,350 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:18:59,356 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:18:59,361 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:18:59,365 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:18:59,370 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:18:59,374 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:18:59,378 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:18:59,382 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:18:59,388 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:18:59,398 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:18:59,431 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:18:59,473 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "C:\\Users\\HASEE\\.conda\\envs\\llamaindex\\lib\\site-packages\\llama_index\\core\\evaluation\\dataset_generation.py:297: DeprecationWarning: Call to deprecated class QueryResponseDataset. (Deprecated in favor of `LabelledRagDataset` which should be used instead.)\n",
      "  return QueryResponseDataset(queries=queries, responses=responses_dict)\n",
      "2025-09-17 16:19:11,361 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0] Original Question: Give me a summary of Toronto\n",
      "[0] Generated Question Variations: ['Of course! Here are 5 related queries that also ask for a summary of Toronto:', '1.  Can you provide a brief overview of the city of Toronto?', \"2.  I'm planning a trip; can you summarize what Toronto is known for?\", \"3.  Give me a high-level description of Toronto's key features and characteristics.\", '4.  What are the most important things to know about Toronto in a nutshell?', \"5.  Please write a concise summary covering Toronto's culture, history, and main attractions.\"]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\HASEE\\AppData\\Local\\Temp\\ipykernel_4504\\719365437.py:30: DeprecationWarning: Call to deprecated class DatasetGenerator. (Deprecated in favor of `RagDatasetGenerator` which should be used instead.)\n",
      "  dataset_generator = DatasetGenerator(\n",
      "2025-09-17 16:19:19,618 - INFO - Retrying request to /chat/completions in 0.498676 seconds\n",
      "2025-09-17 16:19:20,266 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:19:20,270 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:19:20,318 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:19:20,323 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:19:20,341 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:19:20,345 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:19:20,349 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:19:20,354 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:19:20,359 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:19:20,364 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:19:20,372 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:19:20,376 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:19:20,381 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:19:20,386 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:19:20,393 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:19:20,398 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:19:20,402 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:19:20,408 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:19:20,415 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:19:20,603 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "C:\\Users\\HASEE\\.conda\\envs\\llamaindex\\lib\\site-packages\\llama_index\\core\\evaluation\\dataset_generation.py:297: DeprecationWarning: Call to deprecated class QueryResponseDataset. (Deprecated in favor of `LabelledRagDataset` which should be used instead.)\n",
      "  return QueryResponseDataset(queries=queries, responses=responses_dict)\n",
      "2025-09-17 16:19:31,878 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1] Original Question: Give me a summary of Seattle\n",
      "[1] Generated Question Variations: ['1. Provide an overview of Seattle, highlighting its key characteristics.', '2. Can you describe the city of Seattle, including its culture and major landmarks?', \"3. I need a brief summary of Seattle's history and its modern-day identity.\", '4. What are the essential things to know about Seattle in a nutshell?', '5. Give me a quick rundown on Seattle, from its geography to its famous companies.']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\HASEE\\AppData\\Local\\Temp\\ipykernel_4504\\719365437.py:30: DeprecationWarning: Call to deprecated class DatasetGenerator. (Deprecated in favor of `RagDatasetGenerator` which should be used instead.)\n",
      "  dataset_generator = DatasetGenerator(\n",
      "2025-09-17 16:19:38,752 - INFO - Retrying request to /chat/completions in 0.414587 seconds\n",
      "2025-09-17 16:19:39,597 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:19:39,650 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:19:39,717 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:19:39,720 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:19:39,724 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:19:39,728 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:19:39,733 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:19:39,742 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:19:39,746 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:19:39,753 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:19:39,757 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:19:39,762 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:19:39,769 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:19:39,774 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:19:39,778 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:19:39,787 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:19:39,794 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:19:39,801 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:19:39,806 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:19:39,810 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:19:39,820 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:19:39,845 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:19:39,857 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "C:\\Users\\HASEE\\.conda\\envs\\llamaindex\\lib\\site-packages\\llama_index\\core\\evaluation\\dataset_generation.py:297: DeprecationWarning: Call to deprecated class QueryResponseDataset. (Deprecated in favor of `LabelledRagDataset` which should be used instead.)\n",
      "  return QueryResponseDataset(queries=queries, responses=responses_dict)\n",
      "2025-09-17 16:19:51,027 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2] Original Question: Give me a summary of Chicago\n",
      "[2] Generated Question Variations: ['1. Provide an overview of the city of Chicago.', '2. Can you describe the key characteristics of Chicago, from its architecture to its cultural contributions?', '3. Write a brief summary of Chicago for someone unfamiliar with the city.', '4. What are the most important things to know about Chicago in a nutshell?', '5. I need a concise rundown on Chicago, including its history and major attractions.']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\HASEE\\AppData\\Local\\Temp\\ipykernel_4504\\719365437.py:30: DeprecationWarning: Call to deprecated class DatasetGenerator. (Deprecated in favor of `RagDatasetGenerator` which should be used instead.)\n",
      "  dataset_generator = DatasetGenerator(\n",
      "2025-09-17 16:19:59,476 - INFO - Retrying request to /chat/completions in 0.422241 seconds\n",
      "2025-09-17 16:20:00,137 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:00,140 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:00,153 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:00,159 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:00,169 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:00,194 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:00,200 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:00,203 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:00,213 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:00,219 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:00,227 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:00,231 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:00,236 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:00,240 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:00,317 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:00,459 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:00,593 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:00,743 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "C:\\Users\\HASEE\\.conda\\envs\\llamaindex\\lib\\site-packages\\llama_index\\core\\evaluation\\dataset_generation.py:297: DeprecationWarning: Call to deprecated class QueryResponseDataset. (Deprecated in favor of `LabelledRagDataset` which should be used instead.)\n",
      "  return QueryResponseDataset(queries=queries, responses=responses_dict)\n",
      "2025-09-17 16:20:11,848 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3] Original Question: Give me a summary of Boston\n",
      "[3] Generated Question Variations: ['1. Can you provide a brief overview of the city of Boston?', '2. What are the key characteristics and highlights of Boston?', \"3. Describe Boston's history, culture, and major attractions in a nutshell.\", '4. I need a quick synopsis of what Boston is known for.', '5. Summarize the essential aspects of Boston for someone unfamiliar with the city.']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\HASEE\\AppData\\Local\\Temp\\ipykernel_4504\\719365437.py:30: DeprecationWarning: Call to deprecated class DatasetGenerator. (Deprecated in favor of `RagDatasetGenerator` which should be used instead.)\n",
      "  dataset_generator = DatasetGenerator(\n",
      "2025-09-17 16:20:19,769 - INFO - Retrying request to /chat/completions in 0.410763 seconds\n",
      "2025-09-17 16:20:20,844 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:20,861 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:20,917 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:20,921 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:20,926 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:20,931 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:20,937 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:20,941 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:20,959 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:20,965 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:20,970 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:20,975 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:20,981 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:20,988 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:20,992 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:20,998 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:21,004 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:21,009 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:21,015 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:21,024 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:21,034 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:21,085 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:21,091 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "C:\\Users\\HASEE\\.conda\\envs\\llamaindex\\lib\\site-packages\\llama_index\\core\\evaluation\\dataset_generation.py:297: DeprecationWarning: Call to deprecated class QueryResponseDataset. (Deprecated in favor of `LabelledRagDataset` which should be used instead.)\n",
      "  return QueryResponseDataset(queries=queries, responses=responses_dict)\n",
      "2025-09-17 16:20:32,657 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[4] Original Question: Give me a summary of Houston\n",
      "[4] Generated Question Variations: ['1. Provide a brief overview of Houston, Texas.', '2. Can you describe the key characteristics of Houston?', '3. What are the most important things to know about Houston in a nutshell?', \"4. I need a quick summary of Houston's history, culture, and economy.\", '5. Give me the highlights of Houston; what defines the city?']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\HASEE\\AppData\\Local\\Temp\\ipykernel_4504\\719365437.py:30: DeprecationWarning: Call to deprecated class DatasetGenerator. (Deprecated in favor of `RagDatasetGenerator` which should be used instead.)\n",
      "  dataset_generator = DatasetGenerator(\n",
      "2025-09-17 16:20:39,136 - INFO - Retrying request to /chat/completions in 0.389540 seconds\n",
      "2025-09-17 16:20:39,867 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:39,874 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:39,878 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:39,882 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:39,886 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:39,891 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:39,896 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:39,900 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:39,926 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:39,930 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:39,936 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:39,940 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:39,943 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:39,949 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:39,952 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:39,959 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:39,962 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:39,968 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:39,977 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:39,981 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:39,984 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:40,012 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:40,016 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "C:\\Users\\HASEE\\.conda\\envs\\llamaindex\\lib\\site-packages\\llama_index\\core\\evaluation\\dataset_generation.py:297: DeprecationWarning: Call to deprecated class QueryResponseDataset. (Deprecated in favor of `LabelledRagDataset` which should be used instead.)\n",
      "  return QueryResponseDataset(queries=queries, responses=responses_dict)\n",
      "2025-09-17 16:20:51,096 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[5] Original Question: Give me a summary of Tokyo\n",
      "[5] Generated Question Variations: ['1. Provide a brief overview of Tokyo, Japan.', '2. Can you describe the key characteristics of Tokyo, from its culture to its economy?', '3. I need a concise summary of what Tokyo is like.', '4. What are the most important things to know about Tokyo in a nutshell?', \"5. Give me a rundown of Tokyo's main features and attractions.\"]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\HASEE\\AppData\\Local\\Temp\\ipykernel_4504\\719365437.py:30: DeprecationWarning: Call to deprecated class DatasetGenerator. (Deprecated in favor of `RagDatasetGenerator` which should be used instead.)\n",
      "  dataset_generator = DatasetGenerator(\n",
      "2025-09-17 16:20:58,834 - INFO - Retrying request to /chat/completions in 0.412028 seconds\n",
      "2025-09-17 16:20:59,603 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:59,608 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:59,614 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:59,619 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:59,622 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:59,629 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:59,632 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:59,635 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:59,640 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:59,645 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:59,650 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:59,655 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:59,659 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:59,664 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:59,668 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:59,671 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:59,677 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:59,682 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:59,687 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:59,692 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:59,725 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:20:59,750 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "C:\\Users\\HASEE\\.conda\\envs\\llamaindex\\lib\\site-packages\\llama_index\\core\\evaluation\\dataset_generation.py:297: DeprecationWarning: Call to deprecated class QueryResponseDataset. (Deprecated in favor of `LabelledRagDataset` which should be used instead.)\n",
      "  return QueryResponseDataset(queries=queries, responses=responses_dict)\n",
      "2025-09-17 16:21:12,060 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[6] Original Question: Give me a summary of Berlin\n",
      "[6] Generated Question Variations: [\"1. Provide a brief overview of Berlin's history and culture.\", '2. Can you summarize what Berlin is known for?', \"3. I need a concise description of Berlin's key characteristics.\", '4. What are the highlights of Berlin in a nutshell?', '5. Give me a short summary about the city of Berlin.']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\HASEE\\AppData\\Local\\Temp\\ipykernel_4504\\719365437.py:30: DeprecationWarning: Call to deprecated class DatasetGenerator. (Deprecated in favor of `RagDatasetGenerator` which should be used instead.)\n",
      "  dataset_generator = DatasetGenerator(\n",
      "2025-09-17 16:21:18,344 - INFO - Retrying request to /chat/completions in 0.478523 seconds\n",
      "2025-09-17 16:21:19,256 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:21:19,259 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:21:19,265 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:21:19,271 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:21:19,278 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:21:19,299 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:21:19,316 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:21:19,332 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:21:19,338 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:21:19,345 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:21:19,351 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:21:19,364 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:21:19,371 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:21:19,387 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:21:19,397 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:21:19,419 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:21:19,424 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "2025-09-17 16:21:19,469 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n",
      "C:\\Users\\HASEE\\.conda\\envs\\llamaindex\\lib\\site-packages\\llama_index\\core\\evaluation\\dataset_generation.py:297: DeprecationWarning: Call to deprecated class QueryResponseDataset. (Deprecated in favor of `LabelledRagDataset` which should be used instead.)\n",
      "  return QueryResponseDataset(queries=queries, responses=responses_dict)\n",
      "2025-09-17 16:21:30,922 - INFO - HTTP Request: POST https://api.deepseek.com/chat/completions \"HTTP/1.1 200 OK\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[7] Original Question: Give me a summary of Lisbon\n",
      "[7] Generated Question Variations: [\"1. Can you provide an overview of Lisbon's key characteristics?\", '2. What are the main highlights and features of Lisbon to know?', '3. Please describe Lisbon, covering its history, culture, and major attractions.', \"4. I'm unfamiliar with Lisbon; can you give me a brief summary of the city?\", '5. Tell me about Lisbon in a nutshell, from its landmarks to its vibe.']\n"
     ]
    }
   ],
   "execution_count": 12
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-17T08:21:38.047739Z",
     "start_time": "2025-09-17T08:21:38.026707Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# [optional] save\n",
    "dataset.save_json(\"dataset.json\")"
   ],
   "id": "d97b87abd4404295",
   "outputs": [],
   "execution_count": 13
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-17T08:21:38.079736Z",
     "start_time": "2025-09-17T08:21:38.059707Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# [optional] load\n",
    "dataset = EmbeddingQAFinetuneDataset.from_json(\"dataset.json\")"
   ],
   "id": "8a652a6d0f3bb32c",
   "outputs": [],
   "execution_count": 14
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-17T08:21:38.111707Z",
     "start_time": "2025-09-17T08:21:38.091706Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import random\n",
    "\n",
    "\n",
    "def split_train_val_by_query(dataset, split=0.7):\n",
    "    \"\"\"Split dataset by queries.\"\"\"\n",
    "    query_ids = list(dataset.queries.keys())\n",
    "    query_ids_shuffled = random.sample(query_ids, len(query_ids))\n",
    "    split_idx = int(len(query_ids) * split)\n",
    "    train_query_ids = query_ids_shuffled[:split_idx]\n",
    "    eval_query_ids = query_ids_shuffled[split_idx:]\n",
    "\n",
    "    train_queries = {qid: dataset.queries[qid] for qid in train_query_ids}\n",
    "    eval_queries = {qid: dataset.queries[qid] for qid in eval_query_ids}\n",
    "\n",
    "    train_rel_docs = {\n",
    "        qid: dataset.relevant_docs[qid] for qid in train_query_ids\n",
    "    }\n",
    "    eval_rel_docs = {qid: dataset.relevant_docs[qid] for qid in eval_query_ids}\n",
    "\n",
    "    train_dataset = EmbeddingQAFinetuneDataset(\n",
    "        queries=train_queries,\n",
    "        corpus=dataset.corpus,\n",
    "        relevant_docs=train_rel_docs,\n",
    "    )\n",
    "    eval_dataset = EmbeddingQAFinetuneDataset(\n",
    "        queries=eval_queries,\n",
    "        corpus=dataset.corpus,\n",
    "        relevant_docs=eval_rel_docs,\n",
    "    )\n",
    "    return train_dataset, eval_dataset"
   ],
   "id": "9679effb495c7bf9",
   "outputs": [],
   "execution_count": 15
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-17T08:21:38.142703Z",
     "start_time": "2025-09-17T08:21:38.121709Z"
    }
   },
   "cell_type": "code",
   "source": "train_dataset, eval_dataset = split_train_val_by_query(dataset, split=0.7)",
   "id": "f8b18f2e4ce1a460",
   "outputs": [],
   "execution_count": 16
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-17T08:21:40.438594Z",
     "start_time": "2025-09-17T08:21:38.152706Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# generate embeddings dataset\n",
    "from llama_index.finetuning import SentenceTransformersFinetuneEngine"
   ],
   "id": "1cdaad5aacc4acf3",
   "outputs": [],
   "execution_count": 17
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-17T08:24:27.636724Z",
     "start_time": "2025-09-17T08:24:26.263524Z"
    }
   },
   "cell_type": "code",
   "source": [
    "finetune_engine = SentenceTransformersFinetuneEngine(\n",
    "    train_dataset,\n",
    "    model_id=\"D:/pythonProject17/transformers/model_em/BAAI/bge-small-en-v1.5\",\n",
    "    model_output_path=\"test_model3\",\n",
    "    val_dataset=eval_dataset,\n",
    "    epochs=10,  # can set to higher (haven't tested)\n",
    ")"
   ],
   "id": "4a16e465bef0d3fc",
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 16:24:26,290 - INFO - Use pytorch device_name: cuda:0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Use pytorch device_name: cuda:0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 16:24:26,293 - INFO - Load pretrained SentenceTransformer: D:/pythonProject17/transformers/model_em/BAAI/bge-small-en-v1.5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Load pretrained SentenceTransformer: D:/pythonProject17/transformers/model_em/BAAI/bge-small-en-v1.5\n"
     ]
    }
   ],
   "execution_count": 21
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-17T10:46:59.356012Z",
     "start_time": "2025-09-17T10:44:24.679678Z"
    }
   },
   "cell_type": "code",
   "source": "finetune_engine.finetune()",
   "id": "dbbbb68e8b00eaee",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ],
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='480' max='480' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [480/480 02:33, Epoch 10/10]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Cosine Accuracy@1</th>\n",
       "      <th>Cosine Accuracy@3</th>\n",
       "      <th>Cosine Accuracy@5</th>\n",
       "      <th>Cosine Accuracy@10</th>\n",
       "      <th>Cosine Precision@1</th>\n",
       "      <th>Cosine Precision@3</th>\n",
       "      <th>Cosine Precision@5</th>\n",
       "      <th>Cosine Precision@10</th>\n",
       "      <th>Cosine Recall@1</th>\n",
       "      <th>Cosine Recall@3</th>\n",
       "      <th>Cosine Recall@5</th>\n",
       "      <th>Cosine Recall@10</th>\n",
       "      <th>Cosine Ndcg@10</th>\n",
       "      <th>Cosine Mrr@10</th>\n",
       "      <th>Cosine Map@100</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>48</td>\n",
       "      <td>No log</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.793103</td>\n",
       "      <td>0.886700</td>\n",
       "      <td>0.926108</td>\n",
       "      <td>0.965517</td>\n",
       "      <td>0.793103</td>\n",
       "      <td>0.295567</td>\n",
       "      <td>0.185222</td>\n",
       "      <td>0.096552</td>\n",
       "      <td>0.793103</td>\n",
       "      <td>0.886700</td>\n",
       "      <td>0.926108</td>\n",
       "      <td>0.965517</td>\n",
       "      <td>0.878020</td>\n",
       "      <td>0.850047</td>\n",
       "      <td>0.852572</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50</td>\n",
       "      <td>No log</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.793103</td>\n",
       "      <td>0.891626</td>\n",
       "      <td>0.926108</td>\n",
       "      <td>0.965517</td>\n",
       "      <td>0.793103</td>\n",
       "      <td>0.297209</td>\n",
       "      <td>0.185222</td>\n",
       "      <td>0.096552</td>\n",
       "      <td>0.793103</td>\n",
       "      <td>0.891626</td>\n",
       "      <td>0.926108</td>\n",
       "      <td>0.965517</td>\n",
       "      <td>0.878264</td>\n",
       "      <td>0.850381</td>\n",
       "      <td>0.852913</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>96</td>\n",
       "      <td>No log</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.857143</td>\n",
       "      <td>0.940887</td>\n",
       "      <td>0.955665</td>\n",
       "      <td>0.990148</td>\n",
       "      <td>0.857143</td>\n",
       "      <td>0.313629</td>\n",
       "      <td>0.191133</td>\n",
       "      <td>0.099015</td>\n",
       "      <td>0.857143</td>\n",
       "      <td>0.940887</td>\n",
       "      <td>0.955665</td>\n",
       "      <td>0.990148</td>\n",
       "      <td>0.924140</td>\n",
       "      <td>0.902874</td>\n",
       "      <td>0.903769</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>100</td>\n",
       "      <td>No log</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.847291</td>\n",
       "      <td>0.935961</td>\n",
       "      <td>0.955665</td>\n",
       "      <td>0.990148</td>\n",
       "      <td>0.847291</td>\n",
       "      <td>0.311987</td>\n",
       "      <td>0.191133</td>\n",
       "      <td>0.099015</td>\n",
       "      <td>0.847291</td>\n",
       "      <td>0.935961</td>\n",
       "      <td>0.955665</td>\n",
       "      <td>0.990148</td>\n",
       "      <td>0.917558</td>\n",
       "      <td>0.894224</td>\n",
       "      <td>0.895045</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>144</td>\n",
       "      <td>No log</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.862069</td>\n",
       "      <td>0.945813</td>\n",
       "      <td>0.965517</td>\n",
       "      <td>0.995074</td>\n",
       "      <td>0.862069</td>\n",
       "      <td>0.315271</td>\n",
       "      <td>0.193103</td>\n",
       "      <td>0.099507</td>\n",
       "      <td>0.862069</td>\n",
       "      <td>0.945813</td>\n",
       "      <td>0.965517</td>\n",
       "      <td>0.995074</td>\n",
       "      <td>0.929318</td>\n",
       "      <td>0.908238</td>\n",
       "      <td>0.908648</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>150</td>\n",
       "      <td>No log</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.857143</td>\n",
       "      <td>0.940887</td>\n",
       "      <td>0.965517</td>\n",
       "      <td>0.990148</td>\n",
       "      <td>0.857143</td>\n",
       "      <td>0.313629</td>\n",
       "      <td>0.193103</td>\n",
       "      <td>0.099015</td>\n",
       "      <td>0.857143</td>\n",
       "      <td>0.940887</td>\n",
       "      <td>0.965517</td>\n",
       "      <td>0.990148</td>\n",
       "      <td>0.926124</td>\n",
       "      <td>0.905266</td>\n",
       "      <td>0.906087</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>192</td>\n",
       "      <td>No log</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.866995</td>\n",
       "      <td>0.950739</td>\n",
       "      <td>0.965517</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.866995</td>\n",
       "      <td>0.316913</td>\n",
       "      <td>0.193103</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>0.866995</td>\n",
       "      <td>0.950739</td>\n",
       "      <td>0.965517</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.931962</td>\n",
       "      <td>0.910335</td>\n",
       "      <td>0.910335</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>200</td>\n",
       "      <td>No log</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.866995</td>\n",
       "      <td>0.960591</td>\n",
       "      <td>0.970443</td>\n",
       "      <td>0.990148</td>\n",
       "      <td>0.866995</td>\n",
       "      <td>0.320197</td>\n",
       "      <td>0.194089</td>\n",
       "      <td>0.099015</td>\n",
       "      <td>0.866995</td>\n",
       "      <td>0.960591</td>\n",
       "      <td>0.970443</td>\n",
       "      <td>0.990148</td>\n",
       "      <td>0.928896</td>\n",
       "      <td>0.909125</td>\n",
       "      <td>0.909952</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>240</td>\n",
       "      <td>No log</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.871921</td>\n",
       "      <td>0.955665</td>\n",
       "      <td>0.980296</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.871921</td>\n",
       "      <td>0.318555</td>\n",
       "      <td>0.196059</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>0.871921</td>\n",
       "      <td>0.955665</td>\n",
       "      <td>0.980296</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.937123</td>\n",
       "      <td>0.916817</td>\n",
       "      <td>0.916817</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>250</td>\n",
       "      <td>No log</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.866995</td>\n",
       "      <td>0.940887</td>\n",
       "      <td>0.970443</td>\n",
       "      <td>0.995074</td>\n",
       "      <td>0.866995</td>\n",
       "      <td>0.313629</td>\n",
       "      <td>0.194089</td>\n",
       "      <td>0.099507</td>\n",
       "      <td>0.866995</td>\n",
       "      <td>0.940887</td>\n",
       "      <td>0.970443</td>\n",
       "      <td>0.995074</td>\n",
       "      <td>0.930807</td>\n",
       "      <td>0.910028</td>\n",
       "      <td>0.910407</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>288</td>\n",
       "      <td>No log</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.862069</td>\n",
       "      <td>0.955665</td>\n",
       "      <td>0.975369</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.862069</td>\n",
       "      <td>0.318555</td>\n",
       "      <td>0.195074</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>0.862069</td>\n",
       "      <td>0.955665</td>\n",
       "      <td>0.975369</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.930733</td>\n",
       "      <td>0.908269</td>\n",
       "      <td>0.908269</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>300</td>\n",
       "      <td>No log</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.862069</td>\n",
       "      <td>0.950739</td>\n",
       "      <td>0.975369</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.862069</td>\n",
       "      <td>0.316913</td>\n",
       "      <td>0.195074</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>0.862069</td>\n",
       "      <td>0.950739</td>\n",
       "      <td>0.975369</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.931794</td>\n",
       "      <td>0.909618</td>\n",
       "      <td>0.909618</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>336</td>\n",
       "      <td>No log</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.857143</td>\n",
       "      <td>0.960591</td>\n",
       "      <td>0.980296</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.857143</td>\n",
       "      <td>0.320197</td>\n",
       "      <td>0.196059</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>0.857143</td>\n",
       "      <td>0.960591</td>\n",
       "      <td>0.980296</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.930697</td>\n",
       "      <td>0.908023</td>\n",
       "      <td>0.908023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>350</td>\n",
       "      <td>No log</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.866995</td>\n",
       "      <td>0.960591</td>\n",
       "      <td>0.985222</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.866995</td>\n",
       "      <td>0.320197</td>\n",
       "      <td>0.197044</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>0.866995</td>\n",
       "      <td>0.960591</td>\n",
       "      <td>0.985222</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.935927</td>\n",
       "      <td>0.914943</td>\n",
       "      <td>0.914943</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>384</td>\n",
       "      <td>No log</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.862069</td>\n",
       "      <td>0.960591</td>\n",
       "      <td>0.975369</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.862069</td>\n",
       "      <td>0.320197</td>\n",
       "      <td>0.195074</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>0.862069</td>\n",
       "      <td>0.960591</td>\n",
       "      <td>0.975369</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.933807</td>\n",
       "      <td>0.912151</td>\n",
       "      <td>0.912151</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>400</td>\n",
       "      <td>No log</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.862069</td>\n",
       "      <td>0.960591</td>\n",
       "      <td>0.980296</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.862069</td>\n",
       "      <td>0.320197</td>\n",
       "      <td>0.196059</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>0.862069</td>\n",
       "      <td>0.960591</td>\n",
       "      <td>0.980296</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.933742</td>\n",
       "      <td>0.912069</td>\n",
       "      <td>0.912069</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>432</td>\n",
       "      <td>No log</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.871921</td>\n",
       "      <td>0.960591</td>\n",
       "      <td>0.980296</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.871921</td>\n",
       "      <td>0.320197</td>\n",
       "      <td>0.196059</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>0.871921</td>\n",
       "      <td>0.960591</td>\n",
       "      <td>0.980296</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.937378</td>\n",
       "      <td>0.916995</td>\n",
       "      <td>0.916995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>450</td>\n",
       "      <td>No log</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.866995</td>\n",
       "      <td>0.960591</td>\n",
       "      <td>0.980296</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.866995</td>\n",
       "      <td>0.320197</td>\n",
       "      <td>0.196059</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>0.866995</td>\n",
       "      <td>0.960591</td>\n",
       "      <td>0.980296</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.936205</td>\n",
       "      <td>0.915353</td>\n",
       "      <td>0.915353</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>480</td>\n",
       "      <td>No log</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.866995</td>\n",
       "      <td>0.960591</td>\n",
       "      <td>0.980296</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.866995</td>\n",
       "      <td>0.320197</td>\n",
       "      <td>0.196059</td>\n",
       "      <td>0.100000</td>\n",
       "      <td>0.866995</td>\n",
       "      <td>0.960591</td>\n",
       "      <td>0.980296</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.936205</td>\n",
       "      <td>0.915353</td>\n",
       "      <td>0.915353</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ]
     },
     "metadata": {},
     "output_type": "display_data",
     "jetTransient": {
      "display_id": "9c8bc665165da26a50efa4c47ca328bd"
     }
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:44:34,523 - INFO - Information Retrieval Evaluation of the model on the  dataset in epoch 1.0 after 48 steps:\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Information Retrieval Evaluation of the model on the  dataset in epoch 1.0 after 48 steps:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:44:34,794 - INFO - Queries: 203\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Queries: 203\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:44:34,796 - INFO - Corpus: 16\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Corpus: 16\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:44:34,817 - INFO - Score-Function: cosine\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score-Function: cosine\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:44:34,819 - INFO - Accuracy@1: 79.31%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@1: 79.31%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:44:34,832 - INFO - Accuracy@3: 88.67%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@3: 88.67%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:44:34,839 - INFO - Accuracy@5: 92.61%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@5: 92.61%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:44:34,842 - INFO - Accuracy@10: 96.55%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@10: 96.55%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:44:34,848 - INFO - Precision@1: 79.31%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@1: 79.31%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:44:34,852 - INFO - Precision@3: 29.56%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@3: 29.56%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:44:34,856 - INFO - Precision@5: 18.52%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@5: 18.52%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:44:34,860 - INFO - Precision@10: 9.66%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@10: 9.66%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:44:34,864 - INFO - Recall@1: 79.31%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@1: 79.31%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:44:34,867 - INFO - Recall@3: 88.67%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@3: 88.67%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:44:34,872 - INFO - Recall@5: 92.61%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@5: 92.61%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:44:34,877 - INFO - Recall@10: 96.55%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@10: 96.55%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:44:34,883 - INFO - MRR@10: 0.8500\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MRR@10: 0.8500\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:44:34,887 - INFO - NDCG@10: 0.8780\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NDCG@10: 0.8780\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:44:34,890 - INFO - MAP@100: 0.8526\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAP@100: 0.8526\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:44:34,910 - INFO - Save model to test_model3\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Save model to test_model3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:19,370 - INFO - Information Retrieval Evaluation of the model on the  dataset in epoch 1.0416666666666667 after 50 steps:\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Information Retrieval Evaluation of the model on the  dataset in epoch 1.0416666666666667 after 50 steps:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:19,589 - INFO - Queries: 203\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Queries: 203\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:19,592 - INFO - Corpus: 16\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Corpus: 16\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:19,616 - INFO - Score-Function: cosine\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score-Function: cosine\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:19,619 - INFO - Accuracy@1: 79.31%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@1: 79.31%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:19,622 - INFO - Accuracy@3: 89.16%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@3: 89.16%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:19,626 - INFO - Accuracy@5: 92.61%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@5: 92.61%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:19,633 - INFO - Accuracy@10: 96.55%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@10: 96.55%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:19,639 - INFO - Precision@1: 79.31%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@1: 79.31%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:19,643 - INFO - Precision@3: 29.72%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@3: 29.72%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:19,646 - INFO - Precision@5: 18.52%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@5: 18.52%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:19,648 - INFO - Precision@10: 9.66%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@10: 9.66%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:19,651 - INFO - Recall@1: 79.31%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@1: 79.31%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:19,654 - INFO - Recall@3: 89.16%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@3: 89.16%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:19,658 - INFO - Recall@5: 92.61%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@5: 92.61%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:19,661 - INFO - Recall@10: 96.55%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@10: 96.55%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:19,664 - INFO - MRR@10: 0.8504\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MRR@10: 0.8504\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:19,676 - INFO - NDCG@10: 0.8783\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NDCG@10: 0.8783\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:19,687 - INFO - MAP@100: 0.8529\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAP@100: 0.8529\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:19,733 - INFO - Save model to test_model3\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Save model to test_model3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:28,944 - INFO - Information Retrieval Evaluation of the model on the  dataset in epoch 2.0 after 96 steps:\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Information Retrieval Evaluation of the model on the  dataset in epoch 2.0 after 96 steps:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:29,203 - INFO - Queries: 203\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Queries: 203\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:29,206 - INFO - Corpus: 16\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Corpus: 16\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:29,243 - INFO - Score-Function: cosine\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score-Function: cosine\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:29,246 - INFO - Accuracy@1: 85.71%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@1: 85.71%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:29,249 - INFO - Accuracy@3: 94.09%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@3: 94.09%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:29,258 - INFO - Accuracy@5: 95.57%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@5: 95.57%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:29,265 - INFO - Accuracy@10: 99.01%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@10: 99.01%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:29,272 - INFO - Precision@1: 85.71%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@1: 85.71%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:29,276 - INFO - Precision@3: 31.36%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@3: 31.36%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:29,291 - INFO - Precision@5: 19.11%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@5: 19.11%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:29,299 - INFO - Precision@10: 9.90%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@10: 9.90%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:29,310 - INFO - Recall@1: 85.71%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@1: 85.71%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:29,313 - INFO - Recall@3: 94.09%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@3: 94.09%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:29,316 - INFO - Recall@5: 95.57%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@5: 95.57%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:29,319 - INFO - Recall@10: 99.01%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@10: 99.01%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:29,322 - INFO - MRR@10: 0.9029\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MRR@10: 0.9029\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:29,329 - INFO - NDCG@10: 0.9241\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NDCG@10: 0.9241\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:29,339 - INFO - MAP@100: 0.9038\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAP@100: 0.9038\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:29,354 - INFO - Save model to test_model3\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Save model to test_model3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:31,053 - INFO - Information Retrieval Evaluation of the model on the  dataset in epoch 2.0833333333333335 after 100 steps:\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Information Retrieval Evaluation of the model on the  dataset in epoch 2.0833333333333335 after 100 steps:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:31,331 - INFO - Queries: 203\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Queries: 203\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:31,333 - INFO - Corpus: 16\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Corpus: 16\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:31,384 - INFO - Score-Function: cosine\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score-Function: cosine\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:31,389 - INFO - Accuracy@1: 84.73%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@1: 84.73%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:31,404 - INFO - Accuracy@3: 93.60%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@3: 93.60%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:31,422 - INFO - Accuracy@5: 95.57%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@5: 95.57%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:31,441 - INFO - Accuracy@10: 99.01%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@10: 99.01%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:31,447 - INFO - Precision@1: 84.73%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@1: 84.73%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:31,455 - INFO - Precision@3: 31.20%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@3: 31.20%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:31,463 - INFO - Precision@5: 19.11%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@5: 19.11%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:31,468 - INFO - Precision@10: 9.90%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@10: 9.90%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:31,476 - INFO - Recall@1: 84.73%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@1: 84.73%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:31,481 - INFO - Recall@3: 93.60%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@3: 93.60%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:31,487 - INFO - Recall@5: 95.57%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@5: 95.57%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:31,496 - INFO - Recall@10: 99.01%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@10: 99.01%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:31,500 - INFO - MRR@10: 0.8942\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MRR@10: 0.8942\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:31,504 - INFO - NDCG@10: 0.9176\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NDCG@10: 0.9176\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:31,507 - INFO - MAP@100: 0.8950\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAP@100: 0.8950\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:43,073 - INFO - Information Retrieval Evaluation of the model on the  dataset in epoch 3.0 after 144 steps:\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Information Retrieval Evaluation of the model on the  dataset in epoch 3.0 after 144 steps:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:43,414 - INFO - Queries: 203\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Queries: 203\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:43,416 - INFO - Corpus: 16\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Corpus: 16\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:43,448 - INFO - Score-Function: cosine\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score-Function: cosine\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:43,451 - INFO - Accuracy@1: 86.21%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@1: 86.21%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:43,454 - INFO - Accuracy@3: 94.58%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@3: 94.58%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:43,457 - INFO - Accuracy@5: 96.55%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@5: 96.55%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:43,462 - INFO - Accuracy@10: 99.51%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@10: 99.51%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:43,465 - INFO - Precision@1: 86.21%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@1: 86.21%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:43,473 - INFO - Precision@3: 31.53%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@3: 31.53%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:43,477 - INFO - Precision@5: 19.31%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@5: 19.31%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:43,480 - INFO - Precision@10: 9.95%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@10: 9.95%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:43,483 - INFO - Recall@1: 86.21%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@1: 86.21%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:43,486 - INFO - Recall@3: 94.58%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@3: 94.58%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:43,489 - INFO - Recall@5: 96.55%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@5: 96.55%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:43,495 - INFO - Recall@10: 99.51%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@10: 99.51%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:43,498 - INFO - MRR@10: 0.9082\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MRR@10: 0.9082\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:43,502 - INFO - NDCG@10: 0.9293\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NDCG@10: 0.9293\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:43,505 - INFO - MAP@100: 0.9086\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAP@100: 0.9086\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:43,529 - INFO - Save model to test_model3\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Save model to test_model3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:46,150 - INFO - Information Retrieval Evaluation of the model on the  dataset in epoch 3.125 after 150 steps:\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Information Retrieval Evaluation of the model on the  dataset in epoch 3.125 after 150 steps:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:46,447 - INFO - Queries: 203\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Queries: 203\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:46,450 - INFO - Corpus: 16\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Corpus: 16\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:46,484 - INFO - Score-Function: cosine\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score-Function: cosine\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:46,490 - INFO - Accuracy@1: 85.71%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@1: 85.71%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:46,496 - INFO - Accuracy@3: 94.09%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@3: 94.09%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:46,505 - INFO - Accuracy@5: 96.55%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@5: 96.55%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:46,510 - INFO - Accuracy@10: 99.01%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@10: 99.01%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:46,520 - INFO - Precision@1: 85.71%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@1: 85.71%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:46,523 - INFO - Precision@3: 31.36%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@3: 31.36%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:46,528 - INFO - Precision@5: 19.31%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@5: 19.31%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:46,566 - INFO - Precision@10: 9.90%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@10: 9.90%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:46,570 - INFO - Recall@1: 85.71%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@1: 85.71%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:46,575 - INFO - Recall@3: 94.09%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@3: 94.09%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:46,584 - INFO - Recall@5: 96.55%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@5: 96.55%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:46,590 - INFO - Recall@10: 99.01%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@10: 99.01%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:46,595 - INFO - MRR@10: 0.9053\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MRR@10: 0.9053\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:46,600 - INFO - NDCG@10: 0.9261\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NDCG@10: 0.9261\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:46,604 - INFO - MAP@100: 0.9061\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAP@100: 0.9061\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:55,289 - INFO - Information Retrieval Evaluation of the model on the  dataset in epoch 4.0 after 192 steps:\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Information Retrieval Evaluation of the model on the  dataset in epoch 4.0 after 192 steps:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:55,515 - INFO - Queries: 203\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Queries: 203\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:55,516 - INFO - Corpus: 16\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Corpus: 16\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:55,532 - INFO - Score-Function: cosine\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score-Function: cosine\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:55,537 - INFO - Accuracy@1: 86.70%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@1: 86.70%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:55,541 - INFO - Accuracy@3: 95.07%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@3: 95.07%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:55,543 - INFO - Accuracy@5: 96.55%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@5: 96.55%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:55,546 - INFO - Accuracy@10: 100.00%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@10: 100.00%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:55,548 - INFO - Precision@1: 86.70%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@1: 86.70%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:55,551 - INFO - Precision@3: 31.69%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@3: 31.69%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:55,555 - INFO - Precision@5: 19.31%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@5: 19.31%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:55,558 - INFO - Precision@10: 10.00%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@10: 10.00%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:55,562 - INFO - Recall@1: 86.70%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@1: 86.70%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:55,565 - INFO - Recall@3: 95.07%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@3: 95.07%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:55,569 - INFO - Recall@5: 96.55%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@5: 96.55%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:55,572 - INFO - Recall@10: 100.00%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@10: 100.00%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:55,574 - INFO - MRR@10: 0.9103\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MRR@10: 0.9103\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:55,577 - INFO - NDCG@10: 0.9320\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NDCG@10: 0.9320\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:55,580 - INFO - MAP@100: 0.9103\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAP@100: 0.9103\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:55,612 - INFO - Save model to test_model3\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Save model to test_model3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:58,973 - INFO - Information Retrieval Evaluation of the model on the  dataset in epoch 4.166666666666667 after 200 steps:\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Information Retrieval Evaluation of the model on the  dataset in epoch 4.166666666666667 after 200 steps:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:59,201 - INFO - Queries: 203\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Queries: 203\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:59,203 - INFO - Corpus: 16\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Corpus: 16\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:59,221 - INFO - Score-Function: cosine\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score-Function: cosine\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:59,224 - INFO - Accuracy@1: 86.70%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@1: 86.70%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:59,228 - INFO - Accuracy@3: 96.06%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@3: 96.06%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:59,231 - INFO - Accuracy@5: 97.04%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@5: 97.04%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:59,236 - INFO - Accuracy@10: 99.01%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@10: 99.01%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:59,239 - INFO - Precision@1: 86.70%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@1: 86.70%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:59,241 - INFO - Precision@3: 32.02%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@3: 32.02%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:59,247 - INFO - Precision@5: 19.41%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@5: 19.41%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:59,251 - INFO - Precision@10: 9.90%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@10: 9.90%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:59,255 - INFO - Recall@1: 86.70%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@1: 86.70%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:59,258 - INFO - Recall@3: 96.06%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@3: 96.06%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:59,267 - INFO - Recall@5: 97.04%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@5: 97.04%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:59,270 - INFO - Recall@10: 99.01%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@10: 99.01%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:59,274 - INFO - MRR@10: 0.9091\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MRR@10: 0.9091\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:59,277 - INFO - NDCG@10: 0.9289\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NDCG@10: 0.9289\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:45:59,280 - INFO - MAP@100: 0.9100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAP@100: 0.9100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:06,209 - INFO - Information Retrieval Evaluation of the model on the  dataset in epoch 5.0 after 240 steps:\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Information Retrieval Evaluation of the model on the  dataset in epoch 5.0 after 240 steps:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:06,433 - INFO - Queries: 203\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Queries: 203\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:06,434 - INFO - Corpus: 16\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Corpus: 16\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:06,457 - INFO - Score-Function: cosine\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score-Function: cosine\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:06,460 - INFO - Accuracy@1: 87.19%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@1: 87.19%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:06,463 - INFO - Accuracy@3: 95.57%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@3: 95.57%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:06,466 - INFO - Accuracy@5: 98.03%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@5: 98.03%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:06,473 - INFO - Accuracy@10: 100.00%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@10: 100.00%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:06,475 - INFO - Precision@1: 87.19%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@1: 87.19%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:06,479 - INFO - Precision@3: 31.86%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@3: 31.86%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:06,483 - INFO - Precision@5: 19.61%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@5: 19.61%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:06,487 - INFO - Precision@10: 10.00%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@10: 10.00%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:06,492 - INFO - Recall@1: 87.19%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@1: 87.19%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:06,497 - INFO - Recall@3: 95.57%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@3: 95.57%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:06,500 - INFO - Recall@5: 98.03%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@5: 98.03%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:06,504 - INFO - Recall@10: 100.00%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@10: 100.00%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:06,506 - INFO - MRR@10: 0.9168\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MRR@10: 0.9168\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:06,513 - INFO - NDCG@10: 0.9371\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NDCG@10: 0.9371\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:06,516 - INFO - MAP@100: 0.9168\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAP@100: 0.9168\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:06,533 - INFO - Save model to test_model3\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Save model to test_model3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:09,425 - INFO - Information Retrieval Evaluation of the model on the  dataset in epoch 5.208333333333333 after 250 steps:\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Information Retrieval Evaluation of the model on the  dataset in epoch 5.208333333333333 after 250 steps:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:09,660 - INFO - Queries: 203\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Queries: 203\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:09,662 - INFO - Corpus: 16\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Corpus: 16\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:09,686 - INFO - Score-Function: cosine\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score-Function: cosine\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:09,690 - INFO - Accuracy@1: 86.70%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@1: 86.70%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:09,693 - INFO - Accuracy@3: 94.09%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@3: 94.09%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:09,728 - INFO - Accuracy@5: 97.04%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@5: 97.04%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:09,732 - INFO - Accuracy@10: 99.51%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@10: 99.51%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:09,734 - INFO - Precision@1: 86.70%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@1: 86.70%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:09,737 - INFO - Precision@3: 31.36%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@3: 31.36%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:09,739 - INFO - Precision@5: 19.41%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@5: 19.41%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:09,742 - INFO - Precision@10: 9.95%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@10: 9.95%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:09,745 - INFO - Recall@1: 86.70%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@1: 86.70%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:09,781 - INFO - Recall@3: 94.09%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@3: 94.09%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:09,784 - INFO - Recall@5: 97.04%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@5: 97.04%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:09,787 - INFO - Recall@10: 99.51%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@10: 99.51%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:09,791 - INFO - MRR@10: 0.9100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MRR@10: 0.9100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:09,795 - INFO - NDCG@10: 0.9308\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NDCG@10: 0.9308\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:09,802 - INFO - MAP@100: 0.9104\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAP@100: 0.9104\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:16,387 - INFO - Information Retrieval Evaluation of the model on the  dataset in epoch 6.0 after 288 steps:\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Information Retrieval Evaluation of the model on the  dataset in epoch 6.0 after 288 steps:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:16,619 - INFO - Queries: 203\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Queries: 203\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:16,621 - INFO - Corpus: 16\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Corpus: 16\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:16,636 - INFO - Score-Function: cosine\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score-Function: cosine\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:16,639 - INFO - Accuracy@1: 86.21%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@1: 86.21%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:16,642 - INFO - Accuracy@3: 95.57%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@3: 95.57%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:16,648 - INFO - Accuracy@5: 97.54%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@5: 97.54%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:16,652 - INFO - Accuracy@10: 100.00%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@10: 100.00%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:16,655 - INFO - Precision@1: 86.21%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@1: 86.21%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:16,660 - INFO - Precision@3: 31.86%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@3: 31.86%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:16,663 - INFO - Precision@5: 19.51%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@5: 19.51%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:16,666 - INFO - Precision@10: 10.00%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@10: 10.00%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:16,669 - INFO - Recall@1: 86.21%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@1: 86.21%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:16,713 - INFO - Recall@3: 95.57%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@3: 95.57%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:16,720 - INFO - Recall@5: 97.54%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@5: 97.54%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:16,730 - INFO - Recall@10: 100.00%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@10: 100.00%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:16,735 - INFO - MRR@10: 0.9083\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MRR@10: 0.9083\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:16,740 - INFO - NDCG@10: 0.9307\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NDCG@10: 0.9307\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:16,745 - INFO - MAP@100: 0.9083\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAP@100: 0.9083\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:18,853 - INFO - Information Retrieval Evaluation of the model on the  dataset in epoch 6.25 after 300 steps:\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Information Retrieval Evaluation of the model on the  dataset in epoch 6.25 after 300 steps:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:19,079 - INFO - Queries: 203\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Queries: 203\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:19,081 - INFO - Corpus: 16\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Corpus: 16\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:19,097 - INFO - Score-Function: cosine\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score-Function: cosine\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:19,100 - INFO - Accuracy@1: 86.21%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@1: 86.21%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:19,103 - INFO - Accuracy@3: 95.07%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@3: 95.07%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:19,106 - INFO - Accuracy@5: 97.54%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@5: 97.54%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:19,109 - INFO - Accuracy@10: 100.00%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@10: 100.00%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:19,112 - INFO - Precision@1: 86.21%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@1: 86.21%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:19,117 - INFO - Precision@3: 31.69%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@3: 31.69%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:19,120 - INFO - Precision@5: 19.51%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@5: 19.51%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:19,127 - INFO - Precision@10: 10.00%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@10: 10.00%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:19,137 - INFO - Recall@1: 86.21%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@1: 86.21%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:19,140 - INFO - Recall@3: 95.07%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@3: 95.07%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:19,144 - INFO - Recall@5: 97.54%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@5: 97.54%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:19,149 - INFO - Recall@10: 100.00%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@10: 100.00%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:19,154 - INFO - MRR@10: 0.9096\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MRR@10: 0.9096\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:19,156 - INFO - NDCG@10: 0.9318\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NDCG@10: 0.9318\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:19,162 - INFO - MAP@100: 0.9096\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAP@100: 0.9096\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:25,981 - INFO - Information Retrieval Evaluation of the model on the  dataset in epoch 7.0 after 336 steps:\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Information Retrieval Evaluation of the model on the  dataset in epoch 7.0 after 336 steps:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:26,457 - INFO - Queries: 203\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Queries: 203\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:26,461 - INFO - Corpus: 16\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Corpus: 16\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:26,494 - INFO - Score-Function: cosine\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score-Function: cosine\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:26,497 - INFO - Accuracy@1: 85.71%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@1: 85.71%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:26,499 - INFO - Accuracy@3: 96.06%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@3: 96.06%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:26,507 - INFO - Accuracy@5: 98.03%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@5: 98.03%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:26,509 - INFO - Accuracy@10: 100.00%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@10: 100.00%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:26,512 - INFO - Precision@1: 85.71%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@1: 85.71%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:26,514 - INFO - Precision@3: 32.02%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@3: 32.02%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:26,542 - INFO - Precision@5: 19.61%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@5: 19.61%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:26,546 - INFO - Precision@10: 10.00%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@10: 10.00%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:26,550 - INFO - Recall@1: 85.71%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@1: 85.71%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:26,559 - INFO - Recall@3: 96.06%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@3: 96.06%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:26,583 - INFO - Recall@5: 98.03%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@5: 98.03%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:26,591 - INFO - Recall@10: 100.00%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@10: 100.00%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:26,600 - INFO - MRR@10: 0.9080\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MRR@10: 0.9080\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:26,606 - INFO - NDCG@10: 0.9307\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NDCG@10: 0.9307\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:26,615 - INFO - MAP@100: 0.9080\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAP@100: 0.9080\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:31,665 - INFO - Information Retrieval Evaluation of the model on the  dataset in epoch 7.291666666666667 after 350 steps:\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Information Retrieval Evaluation of the model on the  dataset in epoch 7.291666666666667 after 350 steps:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:32,133 - INFO - Queries: 203\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Queries: 203\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:32,137 - INFO - Corpus: 16\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Corpus: 16\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:32,166 - INFO - Score-Function: cosine\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score-Function: cosine\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:32,169 - INFO - Accuracy@1: 86.70%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@1: 86.70%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:32,172 - INFO - Accuracy@3: 96.06%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@3: 96.06%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:32,180 - INFO - Accuracy@5: 98.52%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@5: 98.52%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:32,183 - INFO - Accuracy@10: 100.00%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@10: 100.00%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:32,188 - INFO - Precision@1: 86.70%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@1: 86.70%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:32,192 - INFO - Precision@3: 32.02%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@3: 32.02%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:32,197 - INFO - Precision@5: 19.70%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@5: 19.70%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:32,200 - INFO - Precision@10: 10.00%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@10: 10.00%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:32,204 - INFO - Recall@1: 86.70%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@1: 86.70%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:32,210 - INFO - Recall@3: 96.06%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@3: 96.06%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:32,216 - INFO - Recall@5: 98.52%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@5: 98.52%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:32,224 - INFO - Recall@10: 100.00%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@10: 100.00%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:32,230 - INFO - MRR@10: 0.9149\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MRR@10: 0.9149\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:32,236 - INFO - NDCG@10: 0.9359\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NDCG@10: 0.9359\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:32,240 - INFO - MAP@100: 0.9149\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAP@100: 0.9149\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:38,872 - INFO - Information Retrieval Evaluation of the model on the  dataset in epoch 8.0 after 384 steps:\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Information Retrieval Evaluation of the model on the  dataset in epoch 8.0 after 384 steps:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:39,145 - INFO - Queries: 203\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Queries: 203\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:39,147 - INFO - Corpus: 16\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Corpus: 16\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:39,170 - INFO - Score-Function: cosine\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score-Function: cosine\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:39,174 - INFO - Accuracy@1: 86.21%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@1: 86.21%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:39,176 - INFO - Accuracy@3: 96.06%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@3: 96.06%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:39,179 - INFO - Accuracy@5: 97.54%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@5: 97.54%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:39,185 - INFO - Accuracy@10: 100.00%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@10: 100.00%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:39,190 - INFO - Precision@1: 86.21%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@1: 86.21%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:39,198 - INFO - Precision@3: 32.02%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@3: 32.02%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:39,201 - INFO - Precision@5: 19.51%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@5: 19.51%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:39,204 - INFO - Precision@10: 10.00%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@10: 10.00%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:39,207 - INFO - Recall@1: 86.21%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@1: 86.21%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:39,210 - INFO - Recall@3: 96.06%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@3: 96.06%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:39,213 - INFO - Recall@5: 97.54%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@5: 97.54%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:39,217 - INFO - Recall@10: 100.00%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@10: 100.00%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:39,224 - INFO - MRR@10: 0.9122\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MRR@10: 0.9122\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:39,226 - INFO - NDCG@10: 0.9338\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NDCG@10: 0.9338\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:39,230 - INFO - MAP@100: 0.9122\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAP@100: 0.9122\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:42,122 - INFO - Information Retrieval Evaluation of the model on the  dataset in epoch 8.333333333333334 after 400 steps:\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Information Retrieval Evaluation of the model on the  dataset in epoch 8.333333333333334 after 400 steps:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:42,355 - INFO - Queries: 203\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Queries: 203\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:42,357 - INFO - Corpus: 16\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Corpus: 16\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:42,375 - INFO - Score-Function: cosine\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score-Function: cosine\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:42,378 - INFO - Accuracy@1: 86.21%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@1: 86.21%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:42,381 - INFO - Accuracy@3: 96.06%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@3: 96.06%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:42,384 - INFO - Accuracy@5: 98.03%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@5: 98.03%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:42,387 - INFO - Accuracy@10: 100.00%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@10: 100.00%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:42,390 - INFO - Precision@1: 86.21%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@1: 86.21%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:42,394 - INFO - Precision@3: 32.02%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@3: 32.02%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:42,397 - INFO - Precision@5: 19.61%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@5: 19.61%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:42,407 - INFO - Precision@10: 10.00%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@10: 10.00%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:42,410 - INFO - Recall@1: 86.21%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@1: 86.21%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:42,412 - INFO - Recall@3: 96.06%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@3: 96.06%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:42,415 - INFO - Recall@5: 98.03%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@5: 98.03%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:42,417 - INFO - Recall@10: 100.00%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@10: 100.00%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:42,426 - INFO - MRR@10: 0.9121\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MRR@10: 0.9121\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:42,429 - INFO - NDCG@10: 0.9337\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NDCG@10: 0.9337\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:42,432 - INFO - MAP@100: 0.9121\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAP@100: 0.9121\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:48,382 - INFO - Information Retrieval Evaluation of the model on the  dataset in epoch 9.0 after 432 steps:\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Information Retrieval Evaluation of the model on the  dataset in epoch 9.0 after 432 steps:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:48,615 - INFO - Queries: 203\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Queries: 203\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:48,617 - INFO - Corpus: 16\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Corpus: 16\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:48,635 - INFO - Score-Function: cosine\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score-Function: cosine\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:48,638 - INFO - Accuracy@1: 87.19%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@1: 87.19%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:48,641 - INFO - Accuracy@3: 96.06%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@3: 96.06%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:48,647 - INFO - Accuracy@5: 98.03%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@5: 98.03%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:48,650 - INFO - Accuracy@10: 100.00%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@10: 100.00%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:48,656 - INFO - Precision@1: 87.19%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@1: 87.19%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:48,661 - INFO - Precision@3: 32.02%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@3: 32.02%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:48,663 - INFO - Precision@5: 19.61%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@5: 19.61%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:48,668 - INFO - Precision@10: 10.00%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@10: 10.00%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:48,674 - INFO - Recall@1: 87.19%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@1: 87.19%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:48,676 - INFO - Recall@3: 96.06%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@3: 96.06%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:48,678 - INFO - Recall@5: 98.03%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@5: 98.03%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:48,681 - INFO - Recall@10: 100.00%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@10: 100.00%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:48,684 - INFO - MRR@10: 0.9170\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MRR@10: 0.9170\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:48,691 - INFO - NDCG@10: 0.9374\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NDCG@10: 0.9374\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:48,693 - INFO - MAP@100: 0.9170\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAP@100: 0.9170\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:48,707 - INFO - Save model to test_model3\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Save model to test_model3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:52,720 - INFO - Information Retrieval Evaluation of the model on the  dataset in epoch 9.375 after 450 steps:\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Information Retrieval Evaluation of the model on the  dataset in epoch 9.375 after 450 steps:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:52,955 - INFO - Queries: 203\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Queries: 203\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:52,956 - INFO - Corpus: 16\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Corpus: 16\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:52,972 - INFO - Score-Function: cosine\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score-Function: cosine\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:52,974 - INFO - Accuracy@1: 86.70%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@1: 86.70%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:53,003 - INFO - Accuracy@3: 96.06%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@3: 96.06%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:53,006 - INFO - Accuracy@5: 98.03%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@5: 98.03%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:53,009 - INFO - Accuracy@10: 100.00%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@10: 100.00%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:53,015 - INFO - Precision@1: 86.70%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@1: 86.70%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:53,019 - INFO - Precision@3: 32.02%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@3: 32.02%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:53,023 - INFO - Precision@5: 19.61%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@5: 19.61%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:53,026 - INFO - Precision@10: 10.00%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@10: 10.00%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:53,030 - INFO - Recall@1: 86.70%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@1: 86.70%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:53,033 - INFO - Recall@3: 96.06%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@3: 96.06%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:53,035 - INFO - Recall@5: 98.03%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@5: 98.03%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:53,038 - INFO - Recall@10: 100.00%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@10: 100.00%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:53,040 - INFO - MRR@10: 0.9154\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MRR@10: 0.9154\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:53,043 - INFO - NDCG@10: 0.9362\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NDCG@10: 0.9362\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:53,046 - INFO - MAP@100: 0.9154\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAP@100: 0.9154\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:58,930 - INFO - Information Retrieval Evaluation of the model on the  dataset in epoch 10.0 after 480 steps:\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Information Retrieval Evaluation of the model on the  dataset in epoch 10.0 after 480 steps:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:59,205 - INFO - Queries: 203\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Queries: 203\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:59,208 - INFO - Corpus: 16\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Corpus: 16\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:59,236 - INFO - Score-Function: cosine\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score-Function: cosine\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:59,239 - INFO - Accuracy@1: 86.70%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@1: 86.70%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:59,242 - INFO - Accuracy@3: 96.06%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@3: 96.06%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:59,246 - INFO - Accuracy@5: 98.03%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@5: 98.03%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:59,250 - INFO - Accuracy@10: 100.00%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy@10: 100.00%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:59,253 - INFO - Precision@1: 86.70%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@1: 86.70%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:59,256 - INFO - Precision@3: 32.02%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@3: 32.02%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:59,260 - INFO - Precision@5: 19.61%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@5: 19.61%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:59,272 - INFO - Precision@10: 10.00%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision@10: 10.00%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:59,275 - INFO - Recall@1: 86.70%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@1: 86.70%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:59,279 - INFO - Recall@3: 96.06%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@3: 96.06%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:59,283 - INFO - Recall@5: 98.03%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@5: 98.03%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:59,286 - INFO - Recall@10: 100.00%\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall@10: 100.00%\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:59,289 - INFO - MRR@10: 0.9154\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MRR@10: 0.9154\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:59,296 - INFO - NDCG@10: 0.9362\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NDCG@10: 0.9362\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:46:59,300 - INFO - MAP@100: 0.9154\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MAP@100: 0.9154\n"
     ]
    }
   ],
   "execution_count": 24
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-17T10:47:54.897834Z",
     "start_time": "2025-09-17T10:47:11.617925Z"
    }
   },
   "cell_type": "code",
   "source": "ft_embed_model = finetune_engine.get_finetuned_model()",
   "id": "e1a14538a33e20c2",
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:47:11,709 - INFO - Load pretrained SentenceTransformer: test_model3\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Load pretrained SentenceTransformer: test_model3\n"
     ]
    }
   ],
   "execution_count": 25
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-17T10:48:08.644356Z",
     "start_time": "2025-09-17T10:48:08.616325Z"
    }
   },
   "cell_type": "code",
   "source": "ft_embed_model",
   "id": "7d1dcf476b9ecb5a",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HuggingFaceEmbedding(model_name='test_model3', embed_batch_size=10, callback_manager=<llama_index.core.callbacks.base.CallbackManager object at 0x00000277E7484490>, num_workers=None, embeddings_cache=None, max_length=512, normalize=True, query_instruction=None, text_instruction=None, cache_folder=None, show_progress_bar=False)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 26
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-17T10:49:28.752184Z",
     "start_time": "2025-09-17T10:49:28.725190Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from llama_index.core.selectors import (\n",
    "    EmbeddingSingleSelector,\n",
    "    LLMSingleSelector,\n",
    ")\n",
    "\n",
    "ft_selector = EmbeddingSingleSelector.from_defaults(embed_model=ft_embed_model)\n",
    "base_selector = EmbeddingSingleSelector.from_defaults(\n",
    "    embed_model=base_embed_model\n",
    ")"
   ],
   "id": "44b0a40e885ea4f2",
   "outputs": [],
   "execution_count": 27
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-17T10:50:08.343450Z",
     "start_time": "2025-09-17T10:50:08.315449Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import numpy as np\n",
    "\n",
    "\n",
    "def run_evals(eval_dataset, selector, choices, choice_to_id_dict):\n",
    "    # we just measure accuracy\n",
    "    eval_pairs = eval_dataset.query_docid_pairs\n",
    "    matches = []\n",
    "    for query, relevant_doc_ids in tqdm(eval_pairs):\n",
    "        result = selector.select(choices, query)\n",
    "        # assume single selection for now\n",
    "        pred_doc_id = choice_to_id_dict[result.inds[0]]\n",
    "        gt_doc_id = relevant_doc_ids[0]\n",
    "        matches.append(gt_doc_id == pred_doc_id)\n",
    "    return np.array(matches)"
   ],
   "id": "5b848db460f57e8b",
   "outputs": [],
   "execution_count": 28
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-17T10:52:40.706879Z",
     "start_time": "2025-09-17T10:50:53.887656Z"
    }
   },
   "cell_type": "code",
   "source": "ft_matches = run_evals(eval_dataset, ft_selector, choices, choice_to_id_dict)",
   "id": "13c62620597aaafd",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "  0%|          | 0/203 [00:00<?, ?it/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "bfa059cf96b14eb490c9948e00e0b9a2"
      }
     },
     "metadata": {},
     "output_type": "display_data",
     "jetTransient": {
      "display_id": null
     }
    }
   ],
   "execution_count": 29
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-17T10:55:24.387467Z",
     "start_time": "2025-09-17T10:55:24.363468Z"
    }
   },
   "cell_type": "code",
   "source": "np.mean(ft_matches)",
   "id": "a72d9e3f190073cd",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "np.float64(0.8719211822660099)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 31
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-17T10:55:07.512669Z",
     "start_time": "2025-09-17T10:53:20.357112Z"
    }
   },
   "cell_type": "code",
   "source": [
    "base_matches = run_evals(\n",
    "    eval_dataset, base_selector, choices, choice_to_id_dict\n",
    ")"
   ],
   "id": "ef3932840155f4d2",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "  0%|          | 0/203 [00:00<?, ?it/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "c5700b66a7f343e6bdfd2bc5a43af490"
      }
     },
     "metadata": {},
     "output_type": "display_data",
     "jetTransient": {
      "display_id": null
     }
    }
   ],
   "execution_count": 30
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-17T10:55:39.748278Z",
     "start_time": "2025-09-17T10:55:39.737273Z"
    }
   },
   "cell_type": "code",
   "source": "np.mean(base_matches)",
   "id": "bc57df579ae81ee2",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "np.float64(0.1625615763546798)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 32
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-17T10:57:56.106990Z",
     "start_time": "2025-09-17T10:57:50.800279Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from llama_index.core.query_engine import RouterQueryEngine\n",
    "from llama_index.core import SummaryIndex\n",
    "from llama_index.core import VectorStoreIndex\n",
    "from llama_index.core.tools import QueryEngineTool\n",
    "\n",
    "# define indexes/tools for wikipedia entries\n",
    "tools = []\n",
    "for idx, wiki_title in enumerate(tqdm(wiki_titles)):\n",
    "    doc_id_vector = f\"{wiki_title}_vector\"\n",
    "    doc_id_summary = f\"{wiki_title}_summary\"\n",
    "\n",
    "    vector_index = VectorStoreIndex.from_documents(city_docs[wiki_title])\n",
    "    summary_index = SummaryIndex.from_documents(city_docs[wiki_title])\n",
    "    vector_tool = QueryEngineTool.from_defaults(\n",
    "        query_engine=vector_index.as_query_engine(),\n",
    "        description=city_descs_dict[doc_id_vector],\n",
    "    )\n",
    "    summary_tool = QueryEngineTool.from_defaults(\n",
    "        query_engine=summary_index.as_query_engine(),\n",
    "        description=city_descs_dict[doc_id_summary],\n",
    "    )\n",
    "    tools.extend([vector_tool, summary_tool])"
   ],
   "id": "54d183070795dae0",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "  0%|          | 0/8 [00:00<?, ?it/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "36ba218f6bb64c1aac26c84d77b5d109"
      }
     },
     "metadata": {},
     "output_type": "display_data",
     "jetTransient": {
      "display_id": null
     }
    }
   ],
   "execution_count": 33
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-17T10:58:06.662061Z",
     "start_time": "2025-09-17T10:58:06.640060Z"
    }
   },
   "cell_type": "code",
   "source": [
    "router_query_engine = RouterQueryEngine.from_defaults(\n",
    "    selector=ft_selector.from_defaults(), query_engine_tools=tools\n",
    ")"
   ],
   "id": "4515fc2cd4e3a2d0",
   "outputs": [],
   "execution_count": 34
  },
  {
   "metadata": {
    "jupyter": {
     "is_executing": true
    },
    "ExecuteTime": {
     "start_time": "2025-09-17T10:58:53.529374Z"
    }
   },
   "cell_type": "code",
   "source": [
    "response = router_query_engine.query(\n",
    "    \"Tell me more about the sports teams in Toronto\"\n",
    ")"
   ],
   "id": "c74b1c1479330a0",
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2025-09-17 18:58:54,703 - INFO - Selecting query engine 0: Top similarity match: 0.74, query_engine_tool.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Selecting query engine 0: Top similarity match: 0.74, query_engine_tool.\n"
     ]
    }
   ],
   "execution_count": null
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-09-17T11:28:18.405763Z",
     "start_time": "2025-09-17T11:28:16.794453Z"
    }
   },
   "cell_type": "code",
   "source": "response.source_nodes[0].get_content()",
   "id": "b9d2cf219924fdd0",
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'response' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[1;31mNameError\u001B[0m                                 Traceback (most recent call last)",
      "Cell \u001B[1;32mIn[1], line 1\u001B[0m\n\u001B[1;32m----> 1\u001B[0m \u001B[43mresponse\u001B[49m\u001B[38;5;241m.\u001B[39msource_nodes[\u001B[38;5;241m0\u001B[39m]\u001B[38;5;241m.\u001B[39mget_content()\n",
      "\u001B[1;31mNameError\u001B[0m: name 'response' is not defined"
     ]
    }
   ],
   "execution_count": 1
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "",
   "id": "16adf3beeb089b33"
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
